{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 数据爬取"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from selenium import webdriver\n",
    "import os\n",
    "import requests\n",
    "import pickle\n",
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "product_url = r'https://item.jd.com/5561746.html#none'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "driver = webdriver.Chrome()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "driver.get(product_url)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [],
   "source": [
    "good_comments_url = r'https://club.jd.com/comment/productPageComments.action?callback=fetchJSON_comment98&productId=5561746&score=3&sortType=5&page=0&pageSize=10&isShadowSku=0&fold=1'\n",
    "bad_comments_url = r'https://club.jd.com/comment/productPageComments.action?callback=fetchJSON_comment98&productId=5561746&score=1&sortType=5&page=0&pageSize=10&isShadowSku=0&fold=1'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 182,
   "metadata": {},
   "outputs": [],
   "source": [
    "header = {\n",
    "    'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 11_1_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.114 Safari/537.36',\n",
    "}\n",
    "param = {\n",
    "    'callback': 'fetchJSON_comment98',\n",
    "    'productId': '5561746',\n",
    "    'score': '3',\n",
    "    'sortType': '5',\n",
    "    'page': '',\n",
    "    'pageSize': '10',\n",
    "    'isShadowSku': '0',\n",
    "    'fold': '1',\n",
    "}\n",
    "cookie = {\n",
    "    'Cookie': 'shshshfpa=cb4386b6-c0f2-68a1-b156-2236f499ee30-1590065631; shshshfpb=dXHF9pqH0l8XV8dgbxTlNEQ%3D%3D; __jdu=15900656294791955369661; user-key=3e85b9e4-c7cf-43fd-9e1a-756e9776cab0; cn=0; __jdc=122270672; areaId=19; ipLoc-djd=19-1607-3155-0; shshshfp=83f76a7577a1d3cdcb7f20cd9a99ba87; __jdv=122270672|github.com|-|referral|-|1617588163257; jwotest_product=99; __jda=122270672.15900656294791955369661.1590065629.1617593724.1617596142.26; shshshsID=273af6177249142f26677cc915a91991_2_1617597325824; __jdb=122270672.2.15900656294791955369661|26.1617596142; JSESSIONID=EDBD2A6AD8E18761AFF846595478748F.s1; 3AB9D23F7A4B3C9B=NYA7Y2IYQW7V35YN3PSDHABICJZ5GIKPEEIE6XO7TSEUVYNHVZ7CFQHTY2RYGTGNNEFG2YNVNV5ZYJC36L2IOMHRSM'\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {},
   "outputs": [],
   "source": [
    "session = requests.Session()\n",
    "cookie = requests.utils.cookiejar_from_dict(cookie)\n",
    "session.cookies = cookie\n",
    "res = session.get(url=good_comments_url, headers=header, data=param)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {},
   "outputs": [],
   "source": [
    "good_comments = re.findall(r'fetchJSON_comment98\\((.*)\\)', res.text)[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "str"
      ]
     },
     "execution_count": 155,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(good_comments)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [],
   "source": [
    "good_comments = json.loads(good_comments)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict"
      ]
     },
     "execution_count": 105,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(good_comments)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tqdm import tqdm\n",
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 183,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_good_comments(header, param, cookie, scrap_page=200):\n",
    "    session = requests.Session()\n",
    "    session.cookies = requests.utils.cookiejar_from_dict(cookie)\n",
    "    scrap_page = scrap_page\n",
    "    fail = 0\n",
    "    comments = []\n",
    "    for page in tqdm(range(scrap_page)):\n",
    "        good_comments_url = f'https://club.jd.com/comment/productPageComments.action?callback=fetchJSON_comment98&productId=5561746&score=3&sortType=5&page={page}&pageSize=10&isShadowSku=0&fold=1'\n",
    "        try:\n",
    "            res = session.get(url=good_comments_url, headers=header, data=param)\n",
    "            good_comments = re.findall(r'fetchJSON_comment98\\((.*)\\)', res.text)[0] # 本来获取的是json数据，但是前面加了字符串，所以要删除\n",
    "            good_comments = json.loads(good_comments) # 将字符串转换成字典\n",
    "            for itm in good_comments['comments']:\n",
    "                comments.append(itm['content'])\n",
    "            time.sleep(1)\n",
    "        except Exception as e:\n",
    "            fail += 1\n",
    "            continue\n",
    "    print(f'\\n成功的页数为:{scrap_page - fail}失败的页数为:{fail}')\n",
    "    return comments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 184,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 300/300 [02:59<00:00,  1.67it/s]\n",
      "成功的页数为:119失败的页数为:181\n",
      "\n"
     ]
    }
   ],
   "source": [
    "good_comments = get_good_comments(header=header, param=param, cookie=cookie, scrap_page=150)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 185,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1000"
      ]
     },
     "execution_count": 185,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(good_comments)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 179,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('./data/good_comments.pkl', 'wb') as f:\n",
    "    pickle.dump(good_comments, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "metadata": {},
   "outputs": [],
   "source": [
    "t = pickle.load(open('./data/good_comments.pkl', 'rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_good_comments(header, param, cookie, scrap_page=200):\n",
    "    session = requests.Session()\n",
    "    session.cookies = requests.utils.cookiejar_from_dict(cookie)\n",
    "    scrap_page = scrap_page\n",
    "    fail = 0\n",
    "    comments = []\n",
    "    for page in tqdm(range(scrap_page)):\n",
    "        good_comments_url = f'https://club.jd.com/comment/productPageComments.action'\n",
    "        try:\n",
    "            res = session.get(url=good_comments_url, headers=header, data=param)\n",
    "            good_comments = re.findall(r'fetchJSON_comment98\\((.*)\\)', res.text)[0] # 本来获取的是json数据，但是前面加了字符串，所以要删除\n",
    "            good_comments = json.loads(good_comments) # 将字符串转换成字典\n",
    "            for itm in good_comments['comments']:\n",
    "                comments.append(itm['content'])\n",
    "            time.sleep(1)\n",
    "        except Exception as e:\n",
    "            fail += 1\n",
    "            continue\n",
    "    print(f'\\n成功的页数为:{scrap_page - fail}失败的页数为:{fail}')\n",
    "    return comments"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 数据处理"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import pickle\n",
    "import re\n",
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['good_comments.pkl', 'pool_comments.pkl']"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "os.listdir('./data')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "good_comments = pickle.load(open('./data/good_comments.pkl', 'rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pool_co"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
